
package hyperlinks.crawler;

import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

/**
 *
 * @author madhu
 */
public class Webcrawler extends javax.swing.JFrame {

    /**
     * Creates new form FetchPage
     */
    URL url;
    URLQueue urlqueue=new URLQueue();
    URLQueue completed=new URLQueue();
    ArrayList al=new ArrayList();
    public Webcrawler() {
        initComponents();
    }

    /**
     * This method is called from within the constructor to initialize the form.
     * WARNING: Do NOT modify this code. The content of this method is always
     * regenerated by the Form Editor.
     */
    @SuppressWarnings("unchecked")
    // <editor-fold defaultstate="collapsed" desc="Generated Code">//GEN-BEGIN:initComponents
    private void initComponents() {

        links = new javax.swing.JCheckBox();
        images = new javax.swing.JCheckBox();
        textArea1 = new java.awt.TextArea();
        searchfield = new java.awt.TextField();
        jButton1 = new javax.swing.JButton();
        jButton2 = new javax.swing.JButton();
        jLabel1 = new javax.swing.JLabel();
        imports = new javax.swing.JCheckBox();
        download = new javax.swing.JButton();
        otherurls = new javax.swing.JCheckBox();

        setDefaultCloseOperation(javax.swing.WindowConstants.EXIT_ON_CLOSE);

        links.setText("Links");
        links.setToolTipText("List out Hyperlinks in a webpage");

        images.setText("Images");
        images.setToolTipText("List out media in a Webpage");

        textArea1.setEditable(false);

        searchfield.setCursor(new java.awt.Cursor(java.awt.Cursor.TEXT_CURSOR));
        searchfield.setFont(new java.awt.Font("SansSerif", 2, 14)); // NOI18N
        searchfield.setName("");
        searchfield.setText("http://manabadi.com");
        searchfield.addActionListener(new java.awt.event.ActionListener() {
            public void actionPerformed(java.awt.event.ActionEvent evt) {
                searchfieldActionPerformed(evt);
            }
        });

        jButton1.setText("Click to fetch");
        jButton1.setToolTipText("fetches the url");
        jButton1.addActionListener(new java.awt.event.ActionListener() {
            public void actionPerformed(java.awt.event.ActionEvent evt) {
                jButton1ActionPerformed(evt);
            }
        });

        jButton2.setText("Clear Content");
        jButton2.setToolTipText("Clear the content in text area");
        jButton2.addActionListener(new java.awt.event.ActionListener() {
            public void actionPerformed(java.awt.event.ActionEvent evt) {
                jButton2ActionPerformed(evt);
            }
        });

        jLabel1.setText("      URL:");
        jLabel1.setToolTipText("Enter any valid website url");

        imports.setText("Imports");
        imports.setToolTipText("List out Imports in a webpage");

        download.setText("Download");
        download.setToolTipText("click to download selected checkboxes");
        download.addActionListener(new java.awt.event.ActionListener() {
            public void actionPerformed(java.awt.event.ActionEvent evt) {
                downloadActionPerformed(evt);
            }
        });

        otherurls.setText("New Links");
        otherurls.setToolTipText("links to other websites");

        javax.swing.GroupLayout layout = new javax.swing.GroupLayout(getContentPane());
        getContentPane().setLayout(layout);
        layout.setHorizontalGroup(
            layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
            .addGroup(layout.createSequentialGroup()
                .addGap(22, 22, 22)
                .addComponent(textArea1, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
                .addContainerGap())
            .addGroup(layout.createSequentialGroup()
                .addComponent(jLabel1)
                .addGap(3, 3, 3)
                .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
                    .addGroup(layout.createSequentialGroup()
                        .addComponent(searchfield, javax.swing.GroupLayout.PREFERRED_SIZE, 317, javax.swing.GroupLayout.PREFERRED_SIZE)
                        .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                        .addComponent(jButton1)
                        .addGap(0, 0, Short.MAX_VALUE))
                    .addGroup(layout.createSequentialGroup()
                        .addComponent(links)
                        .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                        .addComponent(images)
                        .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                        .addComponent(imports)
                        .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED)
                        .addComponent(otherurls)
                        .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, 128, Short.MAX_VALUE)
                        .addComponent(jButton2)
                        .addGap(12, 12, 12)
                        .addComponent(download)
                        .addContainerGap())))
        );
        layout.setVerticalGroup(
            layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
            .addGroup(javax.swing.GroupLayout.Alignment.TRAILING, layout.createSequentialGroup()
                .addContainerGap()
                .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING)
                    .addComponent(jButton1)
                    .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
                        .addComponent(jLabel1, javax.swing.GroupLayout.PREFERRED_SIZE, 27, javax.swing.GroupLayout.PREFERRED_SIZE)
                        .addComponent(searchfield, javax.swing.GroupLayout.PREFERRED_SIZE, 27, javax.swing.GroupLayout.PREFERRED_SIZE)))
                .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
                    .addGroup(layout.createSequentialGroup()
                        .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE)
                            .addComponent(links)
                            .addComponent(images)
                            .addComponent(imports)
                            .addComponent(otherurls))
                        .addGap(37, 37, 37))
                    .addGroup(javax.swing.GroupLayout.Alignment.TRAILING, layout.createSequentialGroup()
                        .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE)
                            .addComponent(jButton2)
                            .addComponent(download))
                        .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)))
                .addComponent(textArea1, javax.swing.GroupLayout.DEFAULT_SIZE, 193, Short.MAX_VALUE)
                .addContainerGap())
        );

        pack();
    }// </editor-fold>//GEN-END:initComponents

    private void searchfieldActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_searchfieldActionPerformed
         if("http://".equals(searchfield.getText()) || "".equals(searchfield.getText()))
              append("\nCheck the URL....");
        else
           try {
               setText();
          } catch (MalformedURLException ex) {
            Logger.getLogger(Webcrawler.class.getName()).log(Level.SEVERE, null, ex);
        }
    }//GEN-LAST:event_searchfieldActionPerformed

    /*
     * The following setText() function gets the url from the textfield "URL" in main window
     * and creates a object to FetchPage class and calls 
     * mediaLinks(),hrefLinks(),importLinks() and new urls found..
     * and extracts the recieved Elements objects and appen to text area in for loop
     */
    private void setText() throws MalformedURLException{
    
           String geturl=searchfield.getText();
           url=new URL(geturl);
           if(images.isSelected() || links.isSelected() || imports.isSelected() || otherurls.isSelected())
            {    
              append("Fetching "+url+"...\n");
              FetchPage wc=new  FetchPage(url);
              if(images.isSelected())
                {
                               
                   try{
                        Elements media=wc.mediaLinks();
                        append("\n\n Images("+media.size()+"):\n");
                        for(Element medias : media)
                         {
                           String src=medias.attr("src");
                           append(src);
                         }
                   }catch(Exception inimages){
                          append("No data recieved from "+geturl);
                   }
                 }
           
             if(links.isSelected())
               {
                 ArrayList linkss=wc.hrefLinks();
                 if(!linkss.isEmpty())
                 {   
                   append("\n\n Links("+linkss.size()+"):\n");
                   Iterator itr=linkss.iterator();
                   while(itr.hasNext())
                     {
                         String src=(String) itr.next();
                         append(src);
                     }
                 }
                 else{
                   append("\nNo data recieved from host: "+geturl);
                 }
               }
           
               if(imports.isSelected())
                {
                 try{
                      Elements allimports=wc.importLinks();
                      append("\n\n Imports("+allimports.size()+")");
                     for(Element imported : allimports)
                      {
                        String src=imported.attr("abs:href");
                        append(src);
                      }
                 }catch(Exception inimports){
                    append("\nNo data recieved from "+geturl);
                 }
             }
           
            if(otherurls.isSelected())
              {
                ArrayList<String> allotherurls=wc.otherUrls();   //function call to wc.otherUrls it returns arraylist
                if(!allotherurls.isEmpty())
                {
                  append("\n\n Other urls("+allotherurls.size()+")");
                  urlqueue.clear();  //clear queue first
                  Iterator itr=allotherurls.iterator();  //add arraylist to an iterator
                  while(itr.hasNext())  //until the arraylist is empty
                    {
                      String otherlink=(String) itr.next();
                      append(otherlink);
                      urlqueue.enqueue(otherlink);  // add first page urls to queue             
                    } 
           
                  urlqueue.printQueue(urlqueue);  //just printing all urls in queue
          
                  while(!urlqueue.isEmpty())
                   {
                     String tobefetched=urlqueue.dequeue().toString();
              
                     if(!completed.hasThis(tobefetched))           
                       {
                         completed.enqueue(tobefetched); // adding deleted url to another queue              
                         URL newone =new URL(tobefetched); //making removed element to url
                         FetchPage fetching=new FetchPage(newone);
                         al=fetching.otherUrls();
                         append("\n\n Other urls("+al.size()+")");
                         Iterator itr1=al.iterator();  //add arraylist to an iterator
                         while(itr1.hasNext())  //until the arraylist is empty
                          {
                             String otherlinks=(String) itr1.next();
                             append(otherlinks);
                             urlqueue.enqueue(otherlinks);  //add all urls in each page 
                          }
                          urlqueue.printQueue(completed);
                          append("\n\n\tFetching "+newone+" is Completed");
                          append(completed.size()+""); 
                       }
                     if(completed.size()>=10)
                        break;
                  }
               append(completed.size()+" urls are fetched");
               append("and tobe fetched queue have"+urlqueue.size());
                }
                else {
                  append("\nNo data recieved from host:"+geturl);
                }
             }            
       } // if no checkbox is selected
    else
    {
       append("Please Select any one of the above checkboxes...");
    }    
}
  
    
    
    
    private void jButton1ActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_jButton1ActionPerformed
        
        //if no url  is typed then it prints below message
        if("http://".equals(searchfield.getText()) || "".equals(searchfield.getText()))
             append("\nCheck the URL....");
        else
           try {
              setText();
           } catch (MalformedURLException ex) {
            Logger.getLogger(Webcrawler.class.getName()).log(Level.SEVERE, null, ex);
          }
    }//GEN-LAST:event_jButton1ActionPerformed

    private void jButton2ActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_jButton2ActionPerformed
       textArea1.setText(""); //To clear the content when "clear content button is pressed"
    }//GEN-LAST:event_jButton2ActionPerformed

    private void downloadActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_downloadActionPerformed
        
        append("This action will be added soon");
    }//GEN-LAST:event_downloadActionPerformed
public void append(String urls)
{
textArea1.append(urls+"\n");
}
    /**
     * @param args the command line arguments
     */
    public static void main(String args[]) {
        /*
         * Set the Nimbus look and feel
         */
        //<editor-fold defaultstate="collapsed" desc=" Look and feel setting code (optional) ">
        /*
         * If Nimbus (introduced in Java SE 6) is not available, stay with the
         * default look and feel. For details see
         * http://download.oracle.com/javase/tutorial/uiswing/lookandfeel/plaf.html
         */
        try {
            for (javax.swing.UIManager.LookAndFeelInfo info : javax.swing.UIManager.getInstalledLookAndFeels()) {
                if ("Nimbus".equals(info.getName())) {
                    javax.swing.UIManager.setLookAndFeel(info.getClassName());
                    break;
                }
            }
        } catch (ClassNotFoundException ex) {
            java.util.logging.Logger.getLogger(Webcrawler.class.getName()).log(java.util.logging.Level.SEVERE, null, ex);
        } catch (InstantiationException ex) {
            java.util.logging.Logger.getLogger(Webcrawler.class.getName()).log(java.util.logging.Level.SEVERE, null, ex);
        } catch (IllegalAccessException ex) {
            java.util.logging.Logger.getLogger(Webcrawler.class.getName()).log(java.util.logging.Level.SEVERE, null, ex);
        } catch (javax.swing.UnsupportedLookAndFeelException ex) {
            java.util.logging.Logger.getLogger(Webcrawler.class.getName()).log(java.util.logging.Level.SEVERE, null, ex);
        }
        //</editor-fold>

        /*
         * Create and display the form
         */
        java.awt.EventQueue.invokeLater(new Runnable() {

            public void run() {
                new Webcrawler().setVisible(true);
            }
        });
    }
    // Variables declaration - do not modify//GEN-BEGIN:variables
    private javax.swing.JButton download;
    private javax.swing.JCheckBox images;
    private javax.swing.JCheckBox imports;
    private javax.swing.JButton jButton1;
    private javax.swing.JButton jButton2;
    private javax.swing.JLabel jLabel1;
    private javax.swing.JCheckBox links;
    private javax.swing.JCheckBox otherurls;
    private java.awt.TextField searchfield;
    private java.awt.TextArea textArea1;
    // End of variables declaration//GEN-END:variables
}
